url_list = []

main_page_resp = urllib2.urlopen(main_page_url)
main_page_content = main_page_resp.read()

main_page_file = open(floder+"main_page.html","w+")

#
pattern = re.compile(r'<a href="/problems/.*/">') 

alist = main_page_content.split('\n');

for line in alist:
    
    match1 = pattern.match(line.strip())
    if(match1):
        url_list.append(match1.group())
        pro_des = match1.group()[9:-3]
        new_line = line.replace(pro_des, pro_des[1:]+".html")
        main_page_file.write(new_line+"\n")
    else:
        main_page_file.write(line+"\n")

for i in range(len(url_list)):
    url_list[i] = url_list[i][19:-3]
    
print url_list
    
for url in url_list:
    url_file = open(floder+"problems/"+url+".html","w+")
    url_rsp = urllib2.urlopen(url_head+url)
    url_file.write(url_rsp.read())